{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "95804711",
   "metadata": {},
   "source": [
    "## Train Outlier Detector\n",
    "\n",
    "Based on [Alibi Detect Example](https://docs.seldon.io/projects/alibi-detect/en/stable/examples/od_vae_cifar10.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "97fa4aae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 17:05:36.275254: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-03-09 17:05:36.275268: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "tf.keras.backend.clear_session()\n",
    "from tensorflow.keras.layers import Conv2D, Conv2DTranspose, Dense, Layer, Reshape, InputLayer\n",
    "from tqdm import tqdm\n",
    "\n",
    "from alibi_detect.models.tensorflow import elbo\n",
    "from alibi_detect.od import OutlierVAE\n",
    "from alibi_detect.utils.fetching import fetch_detector\n",
    "from alibi_detect.utils.perturbation import apply_mask\n",
    "from alibi_detect.saving import save_detector, load_detector\n",
    "from alibi_detect.utils.visualize import plot_instance_score, plot_feature_outlier_image\n",
    "\n",
    "logger = tf.get_logger()\n",
    "logger.setLevel(logging.ERROR)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6f421df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3) (50000, 1) (10000, 32, 32, 3) (10000, 1)\n"
     ]
    }
   ],
   "source": [
    "train, test = tf.keras.datasets.cifar10.load_data()\n",
    "X_train, y_train = train\n",
    "X_test, y_test = test\n",
    "\n",
    "X_train = X_train.astype('float32') / 255\n",
    "X_test = X_test.astype('float32') / 255\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b80363be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 17:05:46.653642: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/clive/miniconda3/envs/scv2/lib/python3.9/site-packages/cv2/../../lib64:\n",
      "2023-03-09 17:05:46.653676: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-03-09 17:05:46.653706: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (clive-ThinkPad-P1-Gen-5): /proc/driver/nvidia/version does not exist\n",
      "2023-03-09 17:05:46.654247: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-09 17:05:46.954048: W tensorflow/core/framework/cpu_allocator_impl.cc:82] Allocation of 614400000 exceeds 10% of free system memory.\n",
      "2023-03-09 17:05:47.164088: W tensorflow/core/framework/cpu_allocator_impl.cc:82] Allocation of 614400000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [=] - 123s 157ms/step - loss_ma: 8047.0571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 17:07:50.493324: W tensorflow/core/framework/cpu_allocator_impl.cc:82] Allocation of 614400000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [=] - 121s 154ms/step - loss_ma: -2349.7290\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 17:09:51.538684: W tensorflow/core/framework/cpu_allocator_impl.cc:82] Allocation of 614400000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [=] - 124s 158ms/step - loss_ma: -3547.2254\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-09 17:11:55.269597: W tensorflow/core/framework/cpu_allocator_impl.cc:82] Allocation of 614400000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 64/782 [.] - ETA: 1:50 - loss_ma: -3935.1125"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 29>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m od \u001b[38;5;241m=\u001b[39m OutlierVAE(threshold\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m.015\u001b[39m,  \u001b[38;5;66;03m# threshold for outlier score\u001b[39;00m\n\u001b[1;32m     23\u001b[0m                 score_type\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmse\u001b[39m\u001b[38;5;124m'\u001b[39m,  \u001b[38;5;66;03m# use MSE of reconstruction error for outlier detection\u001b[39;00m\n\u001b[1;32m     24\u001b[0m                 encoder_net\u001b[38;5;241m=\u001b[39mencoder_net,  \u001b[38;5;66;03m# can also pass VAE model instead\u001b[39;00m\n\u001b[1;32m     25\u001b[0m                 decoder_net\u001b[38;5;241m=\u001b[39mdecoder_net,  \u001b[38;5;66;03m# of separate encoder and decoder\u001b[39;00m\n\u001b[1;32m     26\u001b[0m                 latent_dim\u001b[38;5;241m=\u001b[39mlatent_dim,\n\u001b[1;32m     27\u001b[0m                 samples\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m     28\u001b[0m \u001b[38;5;66;03m# train\u001b[39;00m\n\u001b[0;32m---> 29\u001b[0m \u001b[43mod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     30\u001b[0m \u001b[43m        \u001b[49m\u001b[43mloss_fn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43melbo\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     31\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcov_elbo\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mdict\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m.05\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     32\u001b[0m \u001b[43m        \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     33\u001b[0m \u001b[43m        \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;66;03m# save the trained outlier detector\u001b[39;00m\n\u001b[1;32m     36\u001b[0m save_detector(od, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./outlier-detector\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/alibi_detect/od/vae.py:134\u001b[0m, in \u001b[0;36mOutlierVAE.fit\u001b[0;34m(self, X, loss_fn, optimizer, cov_elbo, epochs, batch_size, verbose, log_metric, callbacks)\u001b[0m\n\u001b[1;32m    131\u001b[0m     kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mloss_fn_kwargs\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m {cov_elbo_type: tf\u001b[38;5;241m.\u001b[39mdtypes\u001b[38;5;241m.\u001b[39mcast(cov, tf\u001b[38;5;241m.\u001b[39mfloat32)}\n\u001b[1;32m    133\u001b[0m \u001b[38;5;66;03m# train\u001b[39;00m\n\u001b[0;32m--> 134\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/alibi_detect/models/tensorflow/trainer.py:93\u001b[0m, in \u001b[0;36mtrainer\u001b[0;34m(model, loss_fn, x_train, y_train, dataset, optimizer, loss_fn_kwargs, preprocess_fn, epochs, reg_loss_fn, batch_size, buffer_size, verbose, log_metric, callbacks)\u001b[0m\n\u001b[1;32m     90\u001b[0m         loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(model\u001b[38;5;241m.\u001b[39mlosses)\n\u001b[1;32m     91\u001b[0m     loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m reg_loss_fn(model)  \u001b[38;5;66;03m# alternative way they might be specified\u001b[39;00m\n\u001b[0;32m---> 93\u001b[0m grads \u001b[38;5;241m=\u001b[39m \u001b[43mtape\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgradient\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrainable_weights\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     94\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mapply_gradients(\u001b[38;5;28mzip\u001b[39m(grads, model\u001b[38;5;241m.\u001b[39mtrainable_weights))\n\u001b[1;32m     95\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m verbose:\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/tensorflow/python/eager/backprop.py:1081\u001b[0m, in \u001b[0;36mGradientTape.gradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1077\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_gradients \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1078\u001b[0m   output_gradients \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mif\u001b[39;00m x \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m ops\u001b[38;5;241m.\u001b[39mconvert_to_tensor(x)\n\u001b[1;32m   1079\u001b[0m                       \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m nest\u001b[38;5;241m.\u001b[39mflatten(output_gradients)]\n\u001b[0;32m-> 1081\u001b[0m flat_grad \u001b[38;5;241m=\u001b[39m \u001b[43mimperative_grad\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimperative_grad\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1082\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_tape\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1083\u001b[0m \u001b[43m    \u001b[49m\u001b[43mflat_targets\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1084\u001b[0m \u001b[43m    \u001b[49m\u001b[43mflat_sources\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1085\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_gradients\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_gradients\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1086\u001b[0m \u001b[43m    \u001b[49m\u001b[43msources_raw\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mflat_sources_raw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1087\u001b[0m \u001b[43m    \u001b[49m\u001b[43munconnected_gradients\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43munconnected_gradients\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1089\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_persistent:\n\u001b[1;32m   1090\u001b[0m   \u001b[38;5;66;03m# Keep track of watched variables before setting tape to None\u001b[39;00m\n\u001b[1;32m   1091\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_watched_variables \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tape\u001b[38;5;241m.\u001b[39mwatched_variables()\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/tensorflow/python/eager/imperative_grad.py:67\u001b[0m, in \u001b[0;36mimperative_grad\u001b[0;34m(tape, target, sources, output_gradients, sources_raw, unconnected_gradients)\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m     64\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m     65\u001b[0m       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnknown value for unconnected_gradients: \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m unconnected_gradients)\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_TapeGradient\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     68\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtape\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_tape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# pylint: disable=protected-access\u001b[39;49;00m\n\u001b[1;32m     69\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     70\u001b[0m \u001b[43m    \u001b[49m\u001b[43msources\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     71\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_gradients\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     72\u001b[0m \u001b[43m    \u001b[49m\u001b[43msources_raw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     73\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_str\u001b[49m\u001b[43m(\u001b[49m\u001b[43munconnected_gradients\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/tensorflow/python/eager/backprop.py:156\u001b[0m, in \u001b[0;36m_gradient_function\u001b[0;34m(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads, skip_input_indices, forward_pass_name_scope)\u001b[0m\n\u001b[1;32m    154\u001b[0m     gradient_name_scope \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m forward_pass_name_scope \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    155\u001b[0m   \u001b[38;5;28;01mwith\u001b[39;00m ops\u001b[38;5;241m.\u001b[39mname_scope(gradient_name_scope):\n\u001b[0;32m--> 156\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mgrad_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmock_op\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mout_grads\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    158\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m grad_fn(mock_op, \u001b[38;5;241m*\u001b[39mout_grads)\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/tensorflow/python/ops/math_grad.py:1741\u001b[0m, in \u001b[0;36m_MatMulGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m   1739\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m t_a \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m t_b:\n\u001b[1;32m   1740\u001b[0m   grad_a \u001b[38;5;241m=\u001b[39m gen_math_ops\u001b[38;5;241m.\u001b[39mmat_mul(grad, b, transpose_b\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m-> 1741\u001b[0m   grad_b \u001b[38;5;241m=\u001b[39m \u001b[43mgen_math_ops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmat_mul\u001b[49m\u001b[43m(\u001b[49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtranspose_a\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m t_a \u001b[38;5;129;01mand\u001b[39;00m t_b:\n\u001b[1;32m   1743\u001b[0m   grad_a \u001b[38;5;241m=\u001b[39m gen_math_ops\u001b[38;5;241m.\u001b[39mmat_mul(grad, b)\n",
      "File \u001b[0;32m~/miniconda3/envs/scv2/lib/python3.9/site-packages/tensorflow/python/ops/gen_math_ops.py:6013\u001b[0m, in \u001b[0;36mmat_mul\u001b[0;34m(a, b, transpose_a, transpose_b, name)\u001b[0m\n\u001b[1;32m   6011\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tld\u001b[38;5;241m.\u001b[39mis_eager:\n\u001b[1;32m   6012\u001b[0m   \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 6013\u001b[0m     _result \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_FastPathExecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   6014\u001b[0m \u001b[43m      \u001b[49m\u001b[43m_ctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mMatMul\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtranspose_a\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtranspose_a\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtranspose_b\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   6015\u001b[0m \u001b[43m      \u001b[49m\u001b[43mtranspose_b\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   6016\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _result\n\u001b[1;32m   6017\u001b[0m   \u001b[38;5;28;01mexcept\u001b[39;00m _core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "latent_dim = 1024\n",
    "\n",
    "encoder_net = tf.keras.Sequential(\n",
    "    [\n",
    "        InputLayer(input_shape=(32, 32, 3)),\n",
    "        Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "        Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "        Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu)\n",
    "    ])\n",
    "\n",
    "decoder_net = tf.keras.Sequential(\n",
    "    [\n",
    "        InputLayer(input_shape=(latent_dim,)),\n",
    "        Dense(4*4*128),\n",
    "        Reshape(target_shape=(4, 4, 128)),\n",
    "        Conv2DTranspose(256, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "        Conv2DTranspose(64, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "        Conv2DTranspose(3, 4, strides=2, padding='same', activation='sigmoid')\n",
    "    ])\n",
    "\n",
    "# initialize outlier detector\n",
    "od = OutlierVAE(threshold=.015,  # threshold for outlier score\n",
    "                score_type='mse',  # use MSE of reconstruction error for outlier detection\n",
    "                encoder_net=encoder_net,  # can also pass VAE model instead\n",
    "                decoder_net=decoder_net,  # of separate encoder and decoder\n",
    "                latent_dim=latent_dim,\n",
    "                samples=2)\n",
    "# train\n",
    "od.fit(X_train,\n",
    "        loss_fn=elbo,\n",
    "        cov_elbo=dict(sim=.05),\n",
    "        epochs=50,\n",
    "        verbose=True)\n",
    "\n",
    "# save the trained outlier detector\n",
    "save_detector(od, \"./outlier-detector\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a38614a",
   "metadata": {},
   "source": [
    "Create a MLServer model settings file: `model-settings.json`:\n",
    "\n",
    "```json\n",
    "{\n",
    "  \"name\": \"cifar10-outlier-detect\",\n",
    "  \"implementation\": \"mlserver_alibi_detect.AlibiDetectRuntime\",\n",
    "  \"parameters\": {\n",
    "    \"uri\": \"./\",\n",
    "    \"version\": \"v0.1.0\"\n",
    "  }\n",
    "}\n",
    "\n",
    "```\n",
    "\n",
    "Save to local or remote storage the directory. Here we saved to Google Storage:\n",
    "\n",
    "```bash\n",
    "gsutil ls -R gs://seldon-models/mlserver/alibi-detect/cifar10-outlier \n",
    "\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/:\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/OutlierVAE.dill\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/meta.dill\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model-settings.json\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/:\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/checkpoint\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/decoder_net.h5\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/encoder_net.h5\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/vae.ckpt.data-00000-of-00001\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-outlier/model/vae.ckpt.index\n",
    "\n",
    "```\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785949c4",
   "metadata": {},
   "source": [
    "## Train Drift Detector\n",
    "\n",
    "Based on [Alibi Detect Example](https://docs.seldon.io/projects/alibi-detect/en/stable/examples/cd_ks_cifar10.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "81395f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "\n",
    "from alibi_detect.cd import KSDrift\n",
    "from alibi_detect.models.tensorflow import scale_by_instance\n",
    "from alibi_detect.utils.fetching import fetch_tf_model, fetch_detector\n",
    "from alibi_detect.saving import save_detector, load_detector\n",
    "from alibi_detect.datasets import fetch_cifar10c, corruption_types_cifar10c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5338d603",
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
    "X_train = X_train.astype('float32') / 255\n",
    "X_test = X_test.astype('float32') / 255\n",
    "y_train = y_train.astype('int64').reshape(-1,)\n",
    "y_test = y_test.astype('int64').reshape(-1,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2acb47cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 32, 32, 3) (5000, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "n_test = X_test.shape[0]\n",
    "idx = np.random.choice(n_test, size=n_test // 2, replace=False)\n",
    "idx_h0 = np.delete(np.arange(n_test), idx, axis=0)\n",
    "X_ref,y_ref = X_test[idx], y_test[idx]\n",
    "X_h0, y_h0 = X_test[idx_h0], y_test[idx_h0]\n",
    "print(X_ref.shape, X_h0.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "28d5e887",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Directory drift-detector/preprocess_fn/model does not exist and is now created.\n"
     ]
    }
   ],
   "source": [
    "from functools import partial\n",
    "from tensorflow.keras.layers import Conv2D, Dense, Flatten, InputLayer, Reshape\n",
    "from alibi_detect.cd.tensorflow import preprocess_drift\n",
    "\n",
    "tf.random.set_seed(0)\n",
    "\n",
    "# define encoder\n",
    "encoding_dim = 32\n",
    "encoder_net = tf.keras.Sequential(\n",
    "  [\n",
    "      InputLayer(input_shape=(32, 32, 3)),\n",
    "      Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "      Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "      Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu),\n",
    "      Flatten(),\n",
    "      Dense(encoding_dim,)\n",
    "  ]\n",
    ")\n",
    "\n",
    "# define preprocessing function\n",
    "preprocess_fn = partial(preprocess_drift, model=encoder_net, batch_size=512)\n",
    "\n",
    "# initialise drift detector\n",
    "p_val = .05\n",
    "cd = KSDrift(X_ref, p_val=p_val, preprocess_fn=preprocess_fn)\n",
    "\n",
    "# we can also save/load an initialised detector\n",
    "filepath = 'my_path'  # change to directory where detector is saved\n",
    "save_detector(cd, \"./drift-detector\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeb4314a",
   "metadata": {},
   "source": [
    "Create a MLServer model settings file: `model-settings.json`:\n",
    "\n",
    "```json\n",
    "{\n",
    "  \"name\": \"cifar10-drift\",\n",
    "  \"implementation\": \"mlserver_alibi_detect.AlibiDetectRuntime\",\n",
    "  \"parameters\": {\n",
    "    \"uri\": \"./\",\n",
    "    \"version\": \"v0.1.0\"\n",
    "  }\n",
    "}\n",
    "```\n",
    "\n",
    "Save to local or remote storage the directory. Here we saved to Google Storage:\n",
    "\n",
    "```bash\n",
    "gsutil ls -R gs://seldon-models/mlserver/alibi-detect/cifar10-drift \n",
    "\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/:\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/KSDrift.dill\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/meta.dill\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/model-settings.json\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/model/:\n",
    "gs://seldon-models/mlserver/alibi-detect/cifar10-drift/model/encoder.h5\n",
    "\n",
    "```\n",
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
